package ca.uwindsor.cs.deepweb.utility.documentanalysis;

import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;

import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.TermEnum;

public class TermsAndFrequencyExtractor2 {

	public static void main(String[] args) {

		if (args.length != 2) {
			System.out
					.println("java -jar TermsAndFrequencyExtractor.jar indexDir outputFile");
			System.exit(0);
		}
		try {
			IndexReader reader = IndexReader.open(args[0]);
			System.out.println("NumDocs = " + reader.numDocs());

			ArrayList<TermFrequencyCell> list = new ArrayList<TermFrequencyCell>();

			// Pattern p = Pattern.compile("[,\\d]+");
			// Matcher matcher;
			TermEnum te = reader.terms();
			while (te.next()) {
				// matcher = p.matcher(te.term().text());
				// if (!matcher.find()) {
				TermFrequencyCell cell = new TermFrequencyCell(
						te.term().text(), te.docFreq());
				// line = te.term().text() + "\t" + te.docFreq() + "\n";
				// fp.write(line.getBytes("UTF-8"));
				te.term().text();
				// list.add(te.docFreq());
				list.add(cell);
				// }
			}
			te.close();
			reader.close();
			System.gc();
			Collections.sort(list);
			// Collections.reverse(list);

			FileOutputStream fpy = new FileOutputStream(args[1]);
			String line;
			for (int i = 0; i < list.size(); i++) {
				TermFrequencyCell cell = list.get(i);
				line = cell.getTerm() + "\t" + cell.getFrequency() + "\n";
				fpy.write(line.getBytes());
				fpy.flush();
				// line = Math.log10(i+1) + "\t" + Math.log10(cell.intValue()) +
				// "\n";

			}
			fpy.close();
		} catch (CorruptIndexException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

}
